import scrapy
from gifget.items import GifgetItem

class GifgetSpider(scrapy.Spider):
    # 爬虫的名字
    name = 'gifget_spider'
    # 访问的域名
    domain = 'www.gifget.com'
    # 开始爬虫的页面
    start_url = 'https://www.gifget.com/category/fulihui/page/'
    # 页数
    offset = 1
    # 开始的页面列表
    start_urls = [start_url+str(offset)]

    def parse(self,response):
        # 第一步 爬取开始页的数据
        # 文章标题
        title = response.xpath('//article/header/h2/a/text()').extract()
        # 详情页地址链接
        detailUrl = response.xpath('//article/header/h2/a/@href').extract()
        print(len(detailUrl))
        # 遍历详情页
        for i in range(len(detailUrl)):
            # 存入item
            item = GifgetItem()
            item['title'] = title[i]
            # 调用详情页解析
            yield scrapy.Request(detailUrl[i],self.parse_detail,meta={"item":item})
        self.offset = self.offset+1
        yield scrapy.Request(self.start_url+str(self.offset),self.parse)

    # 解析详情页数据
    def parse_detail(self,response):
        item = response.meta['item']
        # 发布时间
        item['publish_time'] = response.xpath('//div[@class="article-meta"]/span[1]/text()').extract()
        # 图片资源
        item['pics'] = response.xpath('//article[@class="article-content"]/p/img/@src').extract()
        yield item

            
